# -*- coding: utf-8 -*-
'''
Created on Mar 4, 2013

@author: LONG HOANG GIANG
'''

from CrawlerLib2 import commonlib, html2text
from Model.storymodelv3 import StoryModel
import os
import re
import datetime
from urlparse import urljoin
import traceback


class TNL():
    
    _database = '/longhoanggiang/database/{0}'
    _url = ''
    _url_fm = 'http://haynhat.mobi/category/truyen-sex/d-z/page/{0}/'
    
    def __init__(self, database):
        self._database = self._database.format(database)
        
    def getDetail(self, item):
        
        titlefm = "TUYỂN TẬP 18+ HAY NHẤT] - " + commonlib.toUpper(item['title']) + " ({0})"
        page = 1
        chapters = []
        setBreak = False
        url = item['url']
        while not setBreak:
#            url = item['url'] + "{0}/".format(page) if item['url'].endswith('/') else  item['url'] + "/{0}/".format(page)
            tree = commonlib.loadweb(url).build_tree()
            contentNode = tree.xpath("//div[@class='tin-anh']")[0]
            nextNode = tree.xpath("//div[@class='phan-trang']")
            if len(nextNode) <= 0: setBreak = True
            else:
                nextNode = nextNode[0]
                aNodes = nextNode.xpath(".//a")
                firstPage = True if 'Trang' in nextNode.text else False
                for aNode in nextNode.xpath(".//a"):
                    if firstPage:
                        url = aNode.get('href', '')
                        if url == '': setBreak = True
                        firstPage = False
                    else:
                        if aNode.tail == None: 
                            continue
                        if aNode.tail.strip() != '':
                            nextANode = aNode.xpath("./following-sibling::a")
                            if len(nextANode) == 0:
                                setBreak = True
                                break
                            url = nextANode[0].get('href', '')
                            print '??????????????????????????????????'
                            print url
                            print '??????????????????????????????????'
                            if url == '': setBreak = True
            print setBreak
            commonlib.Etree.clean(nextNode)
            commonlib.Etree.clean_elements(contentNode, 'a', {'class': 'ext-link'})
            commonlib.Etree.clean_following_sibling(contentNode.xpath("//h4[contains(., 'Người dùng tìm kiếm')]".decode('utf-8')), True)
            html = html2text.html2text(commonlib.Etree.tostring(contentNode), allow_image=False).encode('utf-8')
            html = html.replace('Game Online / Game Offline / Ứng dụng', '')
            html = html.replace('<h4> </h4>', '').strip()
            html = re.sub(r"\n{3,}", "\n", html)
            print html
            html = re.sub(r"\n", "<br />", html)
            title = titlefm.format(page)
            html = '''<b>{0}</b><br /><hr /><br />{1}'''.format(title, html)
            chapters.append({'title': title, 'detail': html})
            page += 1
        return chapters
        
    def getListCat(self):
        tree = commonlib.loadweb(self._url).build_tree()
        data = []
        for node in tree.xpath("//div[@class='tin-anh']/h3/a"):
            title = commonlib.normalize_str(node.tail)
            href = node.get('href', '')
            print title, href
            data.append({'title': title, 'url': href})
        return data
    
    def process(self, minPage, maxPage):
        model = StoryModel(self._database)
        model.open(True)
        if re.search(r"/\d+/", self._url):
            self._url = re.sub(r"/\d+/$", "", self._url)
        for page in range(minPage, maxPage+1):
            try:
                self._url = self._url_fm.format(page)
                data = self.getListCat()
                for item in data:
                    try:
                        chapters = self.getDetail(item)
                        for chapter in chapters:
                            model.add_story(chapter['title'], chapter['detail'], 0)
                    except:
                        print '______________________________BOQUA______________________'
                        traceback.print_exc()
                        print '______________________________BOQUA______________________'
            except:
                traceback.print_exc()
        model.close()    
        
    
if __name__ == '__main__':
    
    tnl = TNL('tnlhaynhat')
#    tnl.getListCat()    
#    a = tnl.getDetail({'title': '', 'url': 'http://haynhat.mobi/truyen-sex/d-z/cau-chuyen-chua-ai-doc/'})
#    for i in a:
#        print i['title']
#        print '--------------------------------'
#        print i['detail']
    tnl.process(1, 19)
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)